{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analysis of Twitter the-algorithm source code\n",
    "Langchain + Deep Lake with fastchat openai compatible api to analyze the code base of the twitter algorithm. See [langchain code analysis](https://python.langchain.com/en/latest/use_cases/code/twitter-the-algorithm-analysis-deeplake.html) for more details"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!git clone https://github.com/twitter/the-algorithm # replace any repository of your choice "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from langchain.document_loaders import TextLoader\n",
    "\n",
    "root_dir = './the-algorithm'\n",
    "docs = []\n",
    "for dirpath, dirnames, filenames in os.walk(root_dir):\n",
    "    for file in filenames:\n",
    "        try: \n",
    "            loader = TextLoader(os.path.join(dirpath, file), encoding='utf-8')\n",
    "            docs.extend(loader.load_and_split())\n",
    "        except Exception as e: \n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "\n",
    "text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0)\n",
    "texts = text_splitter.split_documents(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import getpass\n",
    "\n",
    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
    "from langchain.vectorstores import DeepLake\n",
    "\n",
    "embeddings = OpenAIEmbeddings()\n",
    "db = DeepLake(embedding_function=embeddings)\n",
    "db.add_documents(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "retriever = db.as_retriever()\n",
    "retriever.search_kwargs['distance_metric'] = 'cos'\n",
    "retriever.search_kwargs['fetch_k'] = 100\n",
    "retriever.search_kwargs['maximal_marginal_relevance'] = True\n",
    "retriever.search_kwargs['k'] = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter(x):\n",
    "    # filter based on source code\n",
    "    if 'com.google' in x['text'].data()['value']:\n",
    "        return False\n",
    "    \n",
    "    # filter based on path e.g. extension\n",
    "    metadata =  x['metadata'].data()['value']\n",
    "    return 'scala' in metadata['source'] or 'py' in metadata['source']\n",
    "\n",
    "### turn on below for custom filtering\n",
    "# retriever.search_kwargs['filter'] = filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.chains import ConversationalRetrievalChain\n",
    "\n",
    "model = ChatOpenAI(model='text-embedding-ada-002') # or name it as gpt4\n",
    "qa = ConversationalRetrievalChain.from_llm(model,retriever=retriever)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions = [\n",
    "    \"What does favCountParams do?\",\n",
    "    \"is it Likes + Bookmarks, or not clear from the code?\",\n",
    "    \"What are the major negative modifiers that lower your linear ranking parameters?\",   \n",
    "    \"How do you get assigned to SimClusters?\",\n",
    "    \"What is needed to migrate from one SimClusters to another SimClusters?\",\n",
    "    \"How much do I get boosted within my cluster?\",   \n",
    "    \"How does Heavy ranker work. what are it’s main inputs?\",\n",
    "    \"How can one influence Heavy ranker?\",\n",
    "    \"why threads and long tweets do so well on the platform?\",\n",
    "    \"Are thread and long tweet creators building a following that reacts to only threads?\",\n",
    "    \"Do you need to follow different strategies to get most followers vs to get most likes and bookmarks per tweet?\",\n",
    "    \"Content meta data and how it impacts virality (e.g. ALT in images).\",\n",
    "    \"What are some unexpected fingerprints for spam factors?\",\n",
    "    \"Is there any difference between company verified checkmarks and blue verified individual checkmarks?\",\n",
    "] \n",
    "chat_history = []\n",
    "\n",
    "for question in questions:  \n",
    "    result = qa({\"question\": question, \"chat_history\": chat_history})\n",
    "    chat_history.append((question, result['answer']))\n",
    "    print(f\"-> **Question**: {question} \\n\")\n",
    "    print(f\"**Answer**: {result['answer']} \\n\")"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
